
                /*********************************************
                **                                          **
                **                                          **
                **              sa.poll.cml.C               **
                **                                          **
                **                                          **
                *********************************************/
                /*********************************************
                **                                          **
                **         Author Jean-Marc Adamo           **
                **       Copyright (C) 1997-2008 JMA        **
                *********************************************/


/*
** Spread Arrays and 
** Non-Blocking Global Read/Write Functions
**Intra Group Communication
** Needs 4 processors
*/

#include "arch.h"

Global_id gid1;
Global_id gid2;
Global_id gid3;
Coll_id cid0, cid1, cid2, cid3, cid4, cid5;

void init_gid(int, char **){
   gid1.set();
   gid2.set();
   gid3.set();
   cid0.set();
   cid1.set();
   cid2.set();
   cid3.set();
   cid4.set();
   cid5.set();
   fprintf(Sdf::trace, "from node %d: gid1.tag=%d, gid2.tag=%d, gid3.tag=%d, cid0.tag=%d, cid1.tag=%d, cid2.tag=%d, cid3.tag=%d, cid4.tag=%d, cid5.tag=%d\n",
               Sdf::PROC_ID, gid1.get(), gid2.get(), gid3.get(), 
               cid0.get(), cid1.get(), cid2.get(), cid3.get(), cid4.get(), cid5.get());
   fflush(Sdf::trace);
}


#define d0 4
#define d1 5
#define d2 2

class RW_count{
   public:
      int cnt;

      RW_count(){
         cnt = 0;
      }
      void reset(){
         cnt = 0;
      }
      void more_RW(int i){
         cnt += i;
      }
};

#define wait_while(condition) while(condition) Sdf::poll();

#define RW_not_complete() (count->cnt < d0*d1*d2)

RW_count *count;

void upon_end_of_RW(HRW_rcd *){
   count->cnt++;
}

class Root_thread: public Thread{
   public:
        Root_thread()
                : Thread((Tmf0_ptr)&Root_thread::body){}

        void body(){

           int (*LA0)[d1][d2];
           int (*LA1)[d1][d2];
           int (*LA2)[d1][d2];

           int colour;
           if(Sdf::PROC_ID==Sdf::PROC_NB-1)
              colour=Sdf::UNDEFINED;
           else
              colour=1;

             //Sdf::UNDEFINED is not counted as a colour
           CommL **comm_table = 
           Sdf::COMML_WORLD->split(cid1, 1, colour, Sdf::PROC_NB-2-Sdf::PROC_ID);
           CommL *comm = *comm_table;

           int comm_rank = comm->rank();
           int comm_size = comm->size();

fprintf(Sdf::trace, "node%d:>>>1 comm=%d, rank=%d, size=%d\n", 
        Sdf::PROC_ID, 
        comm,
        comm_rank,
        comm_size);
fflush(Sdf::trace);

           //access space: my_comm_world
           //definition space: group
           SpreadArray<int> SA0(gid1, d0, d1, d2, 2, 
                                Sdf::COMML_WORLD, comm);
           SpreadArray<int> SA1(gid2, d0, d1, d2, 2, 
                                Sdf::COMML_WORLD, comm);
           SpreadArray<int> SA2(gid3, d0, d1, d2, 2, 
                                Sdf::COMML_WORLD, comm);

fprintf(Sdf::trace, "node%d:>>>SA0 p_id(wrt comm)=%d, offset=%d, comm=%d\n", 
        Sdf::PROC_ID, 
        SA0.get_proc_id(),
        SA0.get_offset(),
        SA0.get_comm());
fflush(Sdf::trace);

fprintf(Sdf::trace, "node%d:>>>SA1 p_id(wrt comm)=%d, offset=%d, comm=%d\n", 
        Sdf::PROC_ID, 
        SA1.get_proc_id(),
        SA1.get_offset(),
        SA1.get_comm());
fflush(Sdf::trace);

fprintf(Sdf::trace, "node%d:>>>SA2 p_id(wrt comm)=%d, offset=%d, comm=%d\n", 
        Sdf::PROC_ID, 
        SA2.get_proc_id(),
        SA2.get_offset(),
        SA2.get_comm());
fflush(Sdf::trace);

           if(Sdf::PROC_ID==Sdf::PROC_NB-1){

              int L0[d0][d1][d2];
              int L1[d0][d1][d2];
              int L2[d0][d1][d2];

              RW_count c;
              count = &c;

              RWCtrl ctrl(upon_end_of_RW, 0);
              SA0.set_ctrl_ptr(&ctrl);
              SA1.set_ctrl_ptr(&ctrl);
              SA2.set_ctrl_ptr(&ctrl);

              Sdf::barrier(cid2);
                //to synchronize the last process in Sdf::COMML_WORLD
                //with the processes in comm before printing
                //writing-reading sequence results.
              fprintf(Sdf::trace, "node%d: !!!!!!!!!!1\n", Sdf::PROC_ID);

              int i0;
              for(i0=0; i0<d0; i0++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++)
                       SA0(i0,j,k).read(L0[i0][j][k]);
              wait_while(RW_not_complete());
              for(i0=0; i0<d0; i0++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       fprintf(Sdf::trace, "node%d: i0=%d, j=%d, k=%d, data=%d\n", 
                               Sdf::PROC_ID, i0, j, k, L0[i0][j][k]);
                       fflush(Sdf::trace);
                    }

              c.reset();
              int i1;
              for(i1=0; i1<d0; i1++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++)
                       SA1(i1,j,k).read(L1[i1][j][k]);
              wait_while(RW_not_complete());
              for(i1=0; i1<d0; i1++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       fprintf(Sdf::trace, "node%d: i1=%d, j=%d, k=%d, data=%d\n", 
                               Sdf::PROC_ID, i1, j, k, L1[i1][j][k]);
                       fflush(Sdf::trace);
                    }

              c.reset();
              int i2;
              for(i2=0; i2<d0; i2++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++)
                       SA2(i2,j,k).read(L2[i2][j][k]);
              wait_while(RW_not_complete());
              for(i2=0; i2<d0; i2++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       fprintf(Sdf::trace, "node%d: i2=%d, j=%d, k=%d, data=%d\n", 
                               Sdf::PROC_ID, i2, j, k, L2[i2][j][k]);
                       fflush(Sdf::trace);
                    }

              Sdf::barrier(cid3);
              return;
           }

           //only processes in comm execute the sequence below
           if(!comm_rank)
              LA0 = new int[d0][d1][d2];
           else
              if(comm_rank == 1)
                 LA1 = new int[d0][d1][d2];
              else
                 LA2 = new int[d0][d1][d2];

           int *local_ptr0 = SA0.to_local_ptr();
           int *local_ptr1 = SA1.to_local_ptr();
           int *local_ptr2 = SA2.to_local_ptr();

           int d = (d0*d1*d2)/comm_size + 1;

           for(int i = 0; i < d; i++){
              local_ptr0[i] = 1000; //to check value change
              local_ptr1[i] = 1001; //to check value change
              local_ptr2[i] = 1002; //to check value change
           }

           Sdf::barrier(cid0, comm);
             //to synchronize global reading-writing sequences
             //with the end of SpreadArray local initialization

           RW_count c;
           count = &c;

           RWCtrl ctrl(upon_end_of_RW, 0);
           SA0.set_ctrl_ptr(&ctrl);
           SA1.set_ctrl_ptr(&ctrl);
           SA2.set_ctrl_ptr(&ctrl);

           if(!comm_rank){
              for(int i1 = 0; i1 < d0; i1++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       SA0(i1, j, k).write(50);
                    }
              wait_while(RW_not_complete());
              Sdf::barrier(cid4, comm);
                //to synchronize subsequent reading with
                //the end of global writing
              count->reset();
              for(int i2 = 0; i2 < d0; i2++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       SA2(i2, j, k).read(LA0[i2][j][k]);
                    }
              wait_while(RW_not_complete());
              Sdf::barrier(cid5, comm);
           }else{
              if(comm_rank == 1){
                 for(int i1 = 0; i1 < d0; i1++)
                    for(int j = 0; j < d1; j++)
                       for(int k = 0; k < d2; k++){
                          SA1(i1, j, k).write(51);
                       }
                 wait_while(RW_not_complete());
                 Sdf::barrier(cid4, comm);
                   //to synchronize subsequent reading with
                   //the end of global writing
                 count->reset();
                 for(int i2 = 0; i2 < d0; i2++)
                    for(int j = 0; j < d1; j++)
                       for(int k = 0; k < d2; k++){
                          SA0(i2, j, k).read(LA1[i2][j][k]);
                       }
                 wait_while(RW_not_complete());
                 Sdf::barrier(cid5, comm);
              }else{ //group_rank == 2
                 for(int i1 = 0; i1 < d0; i1++)
                    for(int j = 0; j < d1; j++)
                       for(int k = 0; k < d2; k++){
                          SA2(i1, j, k).write(52);
                       }
                wait_while(RW_not_complete());
                Sdf::barrier(cid4, comm);
                   //to synchronize subsequent reading with
                   //the end of global writing
                count->reset();
                for(int i2 = 0; i2 < d0; i2++)
                   for(int j = 0; j < d1; j++)
                      for(int k = 0; k < d2; k++){
                         SA1(i2, j, k).read(LA2[i2][j][k]);
                      }
                wait_while(RW_not_complete());
                Sdf::barrier(cid5, comm);
              }
           }
           for(int i1 = 0; i1 < d; i1++){
              fprintf(Sdf::trace, "node%d:local_ptr0=%d,local_ptr0[%d]=%d\n",Sdf::PROC_ID,
                           local_ptr0, i1,  local_ptr0[i1]);
              fprintf(Sdf::trace, "node%d:local_ptr1=%d,local_ptr1[%d]=%d\n",Sdf::PROC_ID,
                           local_ptr1, i1,  local_ptr1[i1]);
              fprintf(Sdf::trace, "node%d:local_ptr2=%d,local_ptr2[%d]=%d\n",Sdf::PROC_ID,
                           local_ptr2, i1,  local_ptr2[i1]);
           }
           for(int i2 = 0; i2 < d0; i2++)
             for(int j = 0; j < d1; j++)
                for(int k = 0; k < d2; k++){
                   if(comm_rank == 0)
                      fprintf(Sdf::trace, "node%d:  LA0[%d][%d][%d] = %d\n", Sdf::PROC_ID, 
                                   i2, j, k,  LA0[i2][j][k]);
                   if(comm_rank == 1)
                      fprintf(Sdf::trace, "node%d:  LA1[%d][%d][%d] = %d\n", Sdf::PROC_ID,
                                  i2, j, k,  LA1[i2][j][k]);
                   if(comm_rank == 2)
                      fprintf(Sdf::trace, "node%d:  LA2[%d][%d][%d] = %d\n", Sdf::PROC_ID,
                                  i2, j, k,  LA2[i2][j][k]);
                }
           fflush(Sdf::trace);

           Sdf::barrier(cid2);
           fprintf(Sdf::trace, "node%d: !!!!!!!!!!2\n", Sdf::PROC_ID);

           Sdf::barrier(cid3);

           delete comm;

           if(!comm_rank)
              delete []LA0;
           else
              if(comm_rank == 1)
                 delete []LA1;
              else
                delete []LA2;

           fprintf(Sdf::trace, "node%d: the end\n\n\n", Sdf::PROC_ID);
           fflush(Sdf::trace);
        }
};

MAIN(Root_thread, init_gid, Sdf::empty, Sdf::empty)
